# coding:utf-8
import threading

from src.core import taskmiddleware, engine
from src.utils import urls
from src.utils.SingletomRedisPool import RedisPool
from src import settings

thread_count = 0


def task(url):
    global thread_count
    r = taskmiddleware.request(url)
    if r:
        urls.insert(url)
        engine.auto_deal_response(r)
    thread_count -= 1


def run():
    global thread_count

    while True:
        try:
            conn = RedisPool.getConn()
            url = conn.lpop(settings.REDIS_QUEUE_URLS)
            if url == None:
                continue
            try:
                if urls.isExist(url):
                    continue
            except TypeError:
                pass
            url = str(url, encoding="utf-8")
            # auto remove duplicate
            if settings.AUTO_REMOVE_DEUP:
                if urls.isExist(url):
                    print(url)
                    continue

            while True:
                if thread_count <= settings.MAX_THREADS_COUNT - 1:
                    t = threading.Thread(target=task, args=(url,))
                    thread_count += 1
                    t.start()
                    break
        except:
            print("The spider has an exception,please inspect it and restart!")


def listen(queue):
    if len(queue) <= 0:
        return False
    else:
        return True
